This notebook is intended for OSeMOSYS users. It allows an OSeMOSYS user to preprocess data files, run models, and visualise results with the click of button.
import pandas as pd
pd.set_option('mode.chained_assignment', None)
#import numpy as np
from IPython.display import HTML
import IPython.core.display as di
#import ipywidgets as widgets
#from ipywidgets import interact, interact_manual
#importing plotly and cufflinks in offline mode
import plotly as py
#import plotly.graph_objs as go
import cufflinks
import plotly.offline as pyo
from plotly.offline import plot, iplot, init_notebook_mode
pyo.init_notebook_mode()
cufflinks.go_offline()
cufflinks.set_config_file(world_readable=True, theme='white')
import os, sys
import subprocess
from tkinter import filedialog
from tkinter import *
from collections import defaultdict
import zipfile
root = Tk()
root.folder = filedialog.askdirectory()
root.destroy()
HTML('''<script>
code_show=true;
function code_toggle() {
if (code_show){
$('div.input').hide();
} else {
$('div.input').show();
}
code_show = !code_show
}
$( document ).ready(code_toggle);
</script>
The raw code for this IPython notebook is by default hidden for easier reading.
To toggle on/off the raw code, click <a href="javascript:code_toggle()">here</a>.''')
#folder_path = r'C:\Users\abhis\Downloads\bolivia_test'
#folder_path = sys.path[0]
data_file = os.path.join(root.folder, 'data.txt') # data_file = os.path.join(sys.path[0], 'data.txt')
glpsol_command = "glpsol --check -m " + '"' + os.path.join(os.getcwd(),'model_new.txt') + '"' + " -d data.txt --wlp cbcinput.lp --log glpsol_log.txt"
cbc_command = "cbc cbcinput.lp solve -solu cbcoutput.txt"
lines = []
with open(data_file, 'r') as f1:
for line in f1:
if not line.startswith(('set MODEper','set MODEx')):
lines.append(line)
with open(data_file, 'w') as f2:
f2.writelines(lines)
parsing = False
data_all = []
data_out = []
data_inp = []
output_table = []
input_table = []
with open(data_file, 'r') as f:
for line in f:
if line.startswith('set YEAR'):
start_year = line.split(' ')[3]
if line.startswith('set COMMODITY'): # Extracts list of COMMODITIES from data file. Some models use FUEL instead.
fuel_list = line.split(' ')[3:-1]
if line.startswith('set FUEL'): # Extracts list of FUELS from data file. Some models use COMMODITIES instead.
fuel_list = line.split(' ')[3:-1]
if line.startswith('set TECHNOLOGY'):
tech_list = line.split(' ')[3:-1]
if line.startswith('set STORAGE'):
storage_list = line.split(' ')[3:-1]
if line.startswith('set MODE_OF_OPERATION'):
mode_list = line.split(' ')[3:-1]
with open(data_file, 'r') as f:
for line in f:
if line.startswith(";"):
parsing = False
if parsing:
if line.startswith('['):
fuel = line.split(',')[2]
tech = line.split(',')[1]
elif line.startswith(start_year):
years = line.rstrip().split(' ')[0:]
years = [i.strip(':=') for i in years]
elif not line.startswith(start_year):
values = line.rstrip().split(' ')[1:]
mode = line.split(' ')[0]
data_out.append(tuple([fuel,tech,mode]))
data_all.append(tuple([tech,mode]))
for i in range(0,len(years)):
output_table.append(tuple([tech,fuel,mode,years[i],values[i]]))
if line.startswith('param OutputActivityRatio'):
parsing = True
with open(data_file, 'r') as f:
for line in f:
if line.startswith(";"):
parsing = False
if parsing:
if line.startswith('['):
fuel = line.split(',')[2]
tech = line.split(',')[1]
elif not line.startswith(start_year):
mode = line.split(' ')[0]
data_inp.append(tuple([fuel,tech,mode]))
data_all.append(tuple([tech,mode]))
for i in range(0,len(years)):
input_table.append(tuple([tech,fuel,mode,years[i],values[i]]))
if line.startswith('param InputActivityRatio'):
parsing = True
# For TechnologyToStorage and TechnologyFromStorage
storage_to = []
storage_from = []
with open(data_file) as f:
for line in f:
if line.startswith(";"):
parsing = False
if parsing:
if line.startswith('['):
tech = line.split(',')[1]
elif not line.startswith(mode_list[0]):
storage = line.split(' ')[0]
values = line.rstrip().split(' ')[1:]
for i in range(0,len(mode_list)):
if values[i] != '0':
storage_to.append(tuple([storage,tech,mode_list[i]]))
#data_all.append(tuple([tech,mode_list[i]]))
if line.startswith('param TechnologyToStorage'):
parsing = True
with open(data_file) as f:
for line in f:
if line.startswith(";"):
parsing = False
if parsing:
if line.startswith('['):
tech = line.split(',')[1]
elif not line.startswith(mode_list[0]):
storage = line.split(' ')[0]
values = line.rstrip().split(' ')[1:]
for i in range(0,len(mode_list)):
if values[i] != '0':
storage_from.append(tuple([storage,tech,mode_list[i]]))
#data_all.append(tuple([tech,mode_list[i]]))
if line.startswith('param TechnologyFromStorage'):
parsing = True
dict_out = defaultdict(list)
dict_inp = defaultdict(list)
dict_all = defaultdict(list)
dict_stt = defaultdict(list)
dict_stf = defaultdict(list)
for f,t,m in data_out:
dict_out[f].append((m,t))
for f,t,m in data_inp:
dict_inp[f].append((m,t))
for t,m in data_all:
if m not in dict_all[t]:
dict_all[t].append(m)
for s,t,m in storage_to:
dict_stt[s].append((m,t))
for s,t,m in storage_from:
dict_stf[s].append((m,t))
# Open data file and delete line with 'end;' statement
lines = []
with open(data_file, 'r') as f1:
for line in f1:
if not line.startswith('end;'):
lines.append(line)
with open(data_file, 'w') as f2:
f2.writelines(lines)
# Append lines at the end of the data file
with open(data_file, 'a') as file_out: # 'a' to open in 'append' mode
for each in fuel_list:
if each in dict_out.keys():
line = 'set MODExTECHNOLOGYperFUELout[' + str(each)+']:=' + str(dict_out[each])
line = line.replace('),',')').replace('[(',' (').replace(')]',')').replace("'","")
else:
line = 'set MODExTECHNOLOGYperFUELout[' + str(each) + ']:='
file_out.write(line + ';' + '\n')
for each in fuel_list:
if each in dict_inp.keys():
line = 'set MODExTECHNOLOGYperFUELin[' + str(each) + ']:=' + str(dict_inp[each])
line = line.replace('),',')').replace('[(',' (').replace(')]',')').replace("'","")
else:
line = 'set MODExTECHNOLOGYperFUELin[' + str(each) + ']:='
file_out.write(line + ';' + '\n')
for each in tech_list:
if each in dict_all.keys():
line = 'set MODEperTECHNOLOGY[' + str(each) + ']:=' + str(dict_all[each]) + '*'
line = line.replace(',','').replace(':=[',':= ').replace(']*','').replace("'","")
else:
line = 'set MODEperTECHNOLOGY[' + str(each) + ']:='
file_out.write(line + ';' + '\n')
if len(storage_list) > 1:
for each in storage_list:
if each in dict_stt.keys():
line = 'set MODExTECHNOLOGYperSTORAGEto[' + str(each)+']:=' + str(dict_out[each])
line = line.replace('),',')').replace('[(',' (').replace(')]',')').replace("'","")
else:
line = 'set MODExTECHNOLOGYperSTORAGEto[' + str(each) + ']:='
file_out.write(line + ';' + '\n')
if len(storage_list) > 1:
for each in storage_list:
if each in dict_stf.keys():
line = 'set MODExTECHNOLOGYperSTORAGEfrom[' + str(each)+']:=' + str(dict_out[each])
line = line.replace('),',')').replace('[(',' (').replace(')]',')').replace("'","")
else:
line = 'set MODExTECHNOLOGYperSTORAGEfrom[' + str(each) + ']:='
file_out.write(line + ';' + '\n')
file_out.write('end;')
try:
os.remove(os.path.join(root.folder, 'cbcoutput.txt'))
except OSError:
pass
#subprocess.Popen('start /wait ' + data_filtering, shell=True, cwd=root.folder).wait()
subprocess.Popen('start /wait ' + glpsol_command, shell=True, cwd=root.folder).wait()
subprocess.Popen('start /wait ' + cbc_command, shell=True, cwd=root.folder).wait()
#for file in os.listdir(root.folder):
# if file.startswith('output'):
# results_file = os.path.join(root.folder, file)
# print(results_file)
results_file = os.path.join(root.folder, 'cbcoutput.txt')
# Create \res\csv folder in current working directory if it doesn't already exist
try:
os.makedirs(os.path.join(root.folder, 'res\csv'))
except FileExistsError:
pass
#Read CBC output file
df = pd.read_csv(results_file, sep='\t')
#if str(df.iloc[0]).split(' ')[0] == "Infeasible":
# print("INFEASIBLE RESULT! CHECK YOUR PARAMETERS!")
# exit(0) # Kill the kernel so we don't continue to run...
if str(df.iloc[0]).split(' ')[0] == "Optimal":
print("Optimal Solution Found.")
df.columns = ['temp']
df['temp'] = df['temp'].str.lstrip(' *\n\t')
df[['temp','value']] = df['temp'].str.split(')', expand=True)
df = df.applymap(lambda x: x.strip() if isinstance(x,str) else x)
df['value'] = df['value'].str.split(' ', expand=True)
df[['parameter','id']] = df['temp'].str.split('(', expand=True)
df['parameter'] = df['parameter'].str.split(' ', expand=True)[1]
df = df.drop('temp', axis=1)
df['value'] = df['value'].astype(float).round(4)
#df.columns = ['temp']
#df[['temp','value']] = df['temp'].str.split(')', expand=True)
#df = df.applymap(lambda x: x.strip() if isinstance(x,str) else x)
#df[['temp','parameter']] = df['temp'].str.split(' ', expand=True)
#df[['parameter','id']] = df['parameter'].str.split('(', expand=True)
##df['value'] = df['value'].str.replace(' 0','')
#df['value'] = df['value'].str.split('\s+').str.get(0)
#df = df.drop('temp', axis=1)
#df['value'] = df['value'].astype(float).round(4)
##df = df[~df['value'].str.contains('e-')]
params = df.parameter.unique()
all_params = {}
cols = {'NewCapacity':['r','t','y'],
'AccumulatedNewCapacity':['r','t','y'],
'TotalCapacityAnnual':['r','t','y'],
'CapitalInvestment':['r','t','y'],
'AnnualVariableOperatingCost':['r','t','y'],
'AnnualFixedOperatingCost':['r','t','y'],
'SalvageValue':['r','t','y'],
'DiscountedSalvageValue':['r','t','y'],
'TotalTechnologyAnnualActivity':['r','t','y'],
'RateOfActivity':['r','l','t','m','y'],
'RateOfTotalActivity':['r','t','l','y'],
'Demand':['r','l','f','y'],
'TotalAnnualTechnologyActivityByMode':['r','t','m','y'],
'TotalTechnologyModelPeriodActivity':['r','t'],
'ProductionByTechnologyAnnual':['r','t','f','y'],
'AnnualTechnologyEmissionByMode':['r','t','e','m','y'],
'AnnualTechnologyEmission':['r','t','e','y'],
'AnnualEmissions':['r','e','y'],
'TotalInputToNewCapacity':['r','f','y'],
'TotalInputToTotalCapacity':['r','f','y'],
'TechnologyActivityChangeByModeCostTotal':['r','t','y']}
for each in params:
df_p = df[df.parameter == each]
df_p[cols[each]] = df_p['id'].str.split(',',expand=True)
cols[each].append('value')
df_p = df_p[cols[each]] # Reorder dataframe to include 'value' as last column
all_params[each] = pd.DataFrame(df_p) # Create a dataframe for each parameter
df_p = df_p.rename(columns={'value':each})
df_p.to_csv(os.path.join(root.folder, 'res\csv', str(each) + '.csv'), index=None) # Print data for each paramter to a CSV file
year_split = []
parsing = False
with open(data_file, 'r') as f:
for line in f:
if line.startswith(";"):
parsing = False
if parsing:
if line.startswith(start_year):
years = line.rstrip().split(' ')[0:]
years = [i.strip(':=') for i in years]
elif not line.startswith(start_year):
time_slice = line.rstrip().split(' ')[0]
values = line.rstrip().split(' ')[1:]
for i in range(0,len(years)):
year_split.append(tuple([time_slice,years[i],values[i]]))
if line.startswith('param YearSplit'):
parsing = True
df_output = pd.DataFrame(output_table, columns=['t','f','m','y','OutputActivityRatio'])
df_yearsplit = pd.DataFrame(year_split, columns=['l','y','YearSplit'])
df_activity = all_params['RateOfActivity'].rename(columns={'value':'RateOfActivity'})
df_out_ys = pd.merge(df_output, df_yearsplit, on='y')
df_out_ys['OutputActivityRatio'] = df_out_ys['OutputActivityRatio'].astype(float)
df_out_ys['YearSplit'] = df_out_ys['YearSplit'].astype(float)
df_prod = pd.merge(df_out_ys, df_activity, on=['t','m','l','y'])
df_prod['ProductionByTechnologyAnnual'] = df_prod['OutputActivityRatio']*df_prod['YearSplit']*df_prod['RateOfActivity']
df_prod = df_prod.drop(['OutputActivityRatio','YearSplit','RateOfActivity'], axis=1)
df_prod = df_prod.groupby(['r','t','f','y'])['ProductionByTechnologyAnnual'].sum().reset_index()
df_prod['ProductionByTechnologyAnnual'] = df_prod['ProductionByTechnologyAnnual'].astype(float).round(4)
df_prod.to_csv(os.path.join(root.folder, 'res\csv', 'ProductionByTechnologyAnnual.csv'), index=None)
all_params['ProductionByTechnologyAnnual'] = df_prod.rename(columns={'ProductionByTechnologyAnnual':'value'})
df_input = pd.DataFrame(input_table, columns=['t','f','m','y','InputActivityRatio'])
df_in_ys = pd.merge(df_input, df_yearsplit, on='y')
df_in_ys['InputActivityRatio'] = df_in_ys['InputActivityRatio'].astype(float)
df_in_ys['YearSplit'] = df_in_ys['YearSplit'].astype(float)
df_use = pd.merge(df_in_ys, df_activity, on=['t','m','l','y'])
df_use['UseByTechnologyAnnual'] = df_use['InputActivityRatio']*df_use['YearSplit']*df_use['RateOfActivity']
df_use = df_use.drop(['InputActivityRatio','YearSplit','RateOfActivity'], axis=1)
df_use = df_use.groupby(['r','t','f','y'])['UseByTechnologyAnnual'].sum().reset_index()
df_use['UseByTechnologyAnnual'] = df_use['UseByTechnologyAnnual'].astype(float).round(4)
df_use.to_csv(os.path.join(root.folder, 'res\csv', 'UseByTechnologyAnnual.csv'), index=None)
all_params['UseByTechnologyAnnual'] = df_use.rename(columns={'UseByTechnologyAnnual':'value'})
# List of columns for detailed energy tables and figures
years = pd.Series([int(i) for i in years])
name_color_codes = pd.read_csv(os.path.join(os.getcwd(),'name_color_codes.csv'), encoding='latin-1')
det_col = dict([(c,n) for c,n in zip(name_color_codes.code, name_color_codes.name_english)])
color_dict = dict([(n,c) for n,c in zip(name_color_codes.name_english, name_color_codes.colour)])
# List of columns for aggregated energy tables and figures
agg_col = {'Coal':['Coal'],
'Oil': ['Diesel','HFO','JFL','Crude oil','Petroleum coke'],
'Gas': ['Natural gas','LNG','LPG'],
'Hydro': ['Hydro'],
'Nuclear': ['Nuclear'],
'Other renewables': ['Biomass','Geothermal','Solar','MSW','Wind'],
'Net electricity imports': ['Net electricity imports']
}
def df_filter(df,lb,ub,t_exclude):
df['t'] = df['t'].str[lb:ub]
df['value'] = df['value'].astype('float64')
df = df[~df['t'].isin(t_exclude)].pivot_table(index='y',
columns='t',
values='value',
aggfunc='sum').reset_index().fillna(0)
df = df.reindex(sorted(df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
new_df = pd.DataFrame()
new_df['y'] = years
new_df['y'] = new_df['y'].astype(int)
df['y'] = df['y'].astype(int)
new_df = pd.merge(new_df,df, how='outer', on='y').fillna(0)
return new_df
def df_plot(df,y_title,p_title):
return df.iplot(x='y',
kind='bar',
barmode='stack',
xTitle='Year',
yTitle=y_title,
color=[color_dict[x] for x in df.columns if x != 'y'],
title=p_title,
showlegend=True)
gfec_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('DEM')].drop('r', axis=1)
gfec_df = gfec_df[~gfec_df.t.str.startswith('DEMAGRSUR')]
gfec_df = gfec_df[~gfec_df.t.str.endswith('CRU')]
gfec_df = df_filter(gfec_df,3,6,['PWR','LVS'])
df_plot(gfec_df,'Energy (PJ)','Gross final energy consumption - by sector')
gfec_fuel_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('DEM')].drop('r', axis=1)
gfec_fuel_df = gfec_fuel_df[~gfec_fuel_df.t.str.startswith('DEMPWR')]
gfec_fuel_df = df_filter(gfec_fuel_df,6,9,['SUR','WND','HYD','SOL','GEO','GWT','CRU'])
df_plot(gfec_fuel_df,'Energy (PJ)','Gross final energy consumption - by fuel')
for sector in ['AGR','COM','IND','OTH','RES','TRA']:
gfec_sector_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('DEM'+sector)].drop('r', axis=1)
gfec_sector_df = df_filter(gfec_sector_df,6,9,['SUR','WND','HYD','SOL','GEO','GWT','CRU'])
df_plot(gfec_sector_df,'Energy (PJ)','Gross final energy consumption - ' + det_col[sector] + ' sector' )
# Power generation capacity (detailed)
cap_df = all_params['TotalCapacityAnnual'][all_params['TotalCapacityAnnual'].t.str.startswith('PWR')].drop('r', axis=1)
cap_df = df_filter(cap_df,3,6,['CNT','TRN','CST','CEN','SOU','NOR'])
df_plot(cap_df,'Gigawatts (GW)','Power Generation Capacity (Detail)')
# Power generation capacity (Aggregated)
cap_agg_df = pd.DataFrame(columns=agg_col)
cap_agg_df.insert(0,'y',cap_df['y'])
cap_agg_df = cap_agg_df.fillna(0.00)
for each in agg_col:
for tech_exists in agg_col[each]:
if tech_exists in cap_df.columns:
cap_agg_df[each] = cap_agg_df[each] + cap_df[tech_exists]
cap_agg_df[each] = cap_agg_df[each].round(2)
cap_agg_df = cap_agg_df.loc[:,(cap_agg_df != 0).any(axis=0)]
df_plot(cap_agg_df,'Gigawatts (GW)','Power Generation Capacity (Aggregate)')
#Power generation (Detailed)
gen_df = all_params['ProductionByTechnologyAnnual'][(all_params['ProductionByTechnologyAnnual'].t.str.startswith('PWR') |
all_params['ProductionByTechnologyAnnual'].t.str.startswith('IMP')) &
all_params['ProductionByTechnologyAnnual'].f.str.startswith('ELC')].drop('r', axis=1)
gen_df = df_filter(gen_df,3,6,['TRN'])
gen_df['Net electricity imports'] = 0
ele_exp_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('EXPELC')].drop('r', axis=1)
if not ele_exp_df.empty:
ele_exp_df = df_filter(ele_exp_df,3,6,['TRN']).rename(columns={'Electricity':'Electricity exports'})
gen_df = gen_df.merge(ele_exp_df)
gen_df['Net electricity imports'] = gen_df['Net electricity imports'] - gen_df['Electricity exports']
gen_df.drop('Electricity exports', axis=1, inplace=True)
if 'Electricity' in gen_df.columns:
gen_df['Net electricity imports'] = gen_df['Net electricity imports'] - gen_df['Electricity']
gen_df.drop('Electricity', axis=1, inplace=True)
#gen_df = gen_df.rename(columns={'Electricity':'Electricity imports'})
#df_plot(gen_df,'Petajoules (PJ)','Power Generation (Detail)')
gen_df.iplot(x='y',
kind='bar',
barmode='relative',
xTitle='Year',
yTitle='Petajoules (PJ)',
color=[color_dict[x] for x in gen_df.columns if x != 'y'],
title='Power Generation (Detail)',
showlegend=True)
# Power generation (Aggregated)
gen_agg_df = pd.DataFrame(columns=agg_col)
gen_agg_df.insert(0,'y',gen_df['y'])
gen_agg_df = gen_agg_df.fillna(0.00)
for each in agg_col:
for tech_exists in agg_col[each]:
if tech_exists in gen_df.columns:
gen_agg_df[each] = gen_agg_df[each] + gen_df[tech_exists]
gen_agg_df[each] = gen_agg_df[each].round(2)
gen_agg_df = gen_agg_df.loc[:,(gen_agg_df != 0).any(axis=0)]
gen_agg_df.iplot(x='y',
kind='bar',
barmode='relative',
xTitle='Year',
yTitle='Petajoules (PJ)',
color=[color_dict[x] for x in gen_agg_df.columns if x != 'y'],
title='Power Generation (Aggregate)',
showlegend=True)
# Fuel use for power generation
gen_use_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].t.str.startswith('DEMPWR')].drop('r', axis=1)
gen_use_df = df_filter(gen_use_df,6,9,['SUR'])
df_plot(gen_use_df,'Petajoules (PJ)','Power Generation (Fuel use)')
#Power sector emissions
emissions_df = all_params['AnnualTechnologyEmission'][all_params['AnnualTechnologyEmission'].t.str.startswith('DEM')].drop('r', axis=1)
emissions_fuel_df = df_filter(emissions_df,6,9,[])
emissions_df = all_params['AnnualTechnologyEmission'][all_params['AnnualTechnologyEmission'].t.str.startswith('DEM')].drop('r', axis=1)
emissions_sector_df = df_filter(emissions_df,3,6,[])
df_plot(emissions_fuel_df,'Million tonnes of CO2','CO2 emissions by fuel')
df_plot(emissions_sector_df,'Million tonnes of CO2','CO2 emissions by sector')
#Domestic fuel production
dom_prd_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].t.str.startswith('MIN')|
all_params['ProductionByTechnologyAnnual'].t.str.startswith('RNW')].drop('r', axis=1)
dom_prd_df = df_filter(dom_prd_df,3,6,[])
for each in dom_prd_df.columns:
if each in ['Land','Water', 'Precipitation']: #,'Hydro','Solar','Wind','Geothermal'
dom_prd_df = dom_prd_df.drop(each, axis=1)
try:
df_plot(dom_prd_df,'Petajoules (PJ)','Domestic energy production')
except:
print("No domestic fuel production.")
#Energy imports
ene_imp_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].t.str.startswith('IMP')].drop('r', axis=1)
ene_imp_df = df_filter(ene_imp_df,3,6,[])
if not ene_imp_df.empty:
df_plot(ene_imp_df,'Petajoules (PJ)','Energy imports')
#Energy exports
ene_exp_df = all_params['TotalTechnologyAnnualActivity'][all_params['TotalTechnologyAnnualActivity'].t.str.startswith('EXP')].drop('r', axis=1)
ene_exp_df = df_filter(ene_exp_df,3,6,[])
if len(ene_exp_df.columns) > 1:
df_plot(ene_exp_df,'Petajoules (PJ)','Energy exports')
cap_cos_df = all_params['CapitalInvestment'][all_params['CapitalInvestment'].t.str.startswith('PWR')].drop('r', axis=1)
cap_cos_df = df_filter(cap_cos_df,3,6,['TRN'])
df_plot(cap_cos_df,'Million $','Capital Investment')
ele_cos_df = pd.DataFrame(columns=['Total capital investment', 'Capital costs'])
ele_cos_df.insert(0,'y',years)
ele_cos_df['Total capital investment'] = cap_cos_df.iloc[:,1:].sum(axis=1)
ele_cos_df['Capital costs'] = 0
ele_cos_df = ele_cos_df.fillna(0)
R = 0.1 # Discount rate
n = 30 # Amortization period
cap_exist_total = 500 # Payments on existing capacity (legacy costs)
for i in ele_cos_df['y']:
for j in ele_cos_df['y']:
if i < j + n and i >= j:
ele_cos_df.loc[ele_cos_df['y']==i,'Capital costs'] = ele_cos_df.loc[ele_cos_df['y']==i,'Capital costs'] + (((ele_cos_df.loc[ele_cos_df['y']==j,'Total capital investment'].iloc[0])*R)/(1-(1+R)**(-n)))
ele_cos_df.drop('Total capital investment', axis=1, inplace=True)
cap_exist_values = {}
for year in years:
if cap_exist_total - ((cap_exist_total/n)*(year - int(start_year))) > 0:
cap_exist_values[year] = cap_exist_total - ((cap_exist_total/n)*(year - int(start_year)))
else:
cap_exist_values[year] = 0
ele_cos_df['Legacy costs'] = ele_cos_df['y'].map(cap_exist_values)
ele_cos_df['Capital costs'] += ele_cos_df['Legacy costs']
ele_cos_df = ele_cos_df.drop('Legacy costs', axis=1)
fix_cos_df = all_params['AnnualFixedOperatingCost'][all_params['AnnualFixedOperatingCost'].t.str.startswith('PWR')].drop('r', axis=1)
fix_cos_df = df_filter(fix_cos_df,3,6,['TRN'])
var_cos_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('PWR')|
all_params['AnnualVariableOperatingCost'].t.str.startswith('EXPELC')].drop('r', axis=1)
var_cos_df = df_filter(var_cos_df,3,6,['TRN'])
dis_cos_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('DEMPWR')].drop('r', axis=1)
dis_cos_df = df_filter(dis_cos_df,6,9,[])
dom_val_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('MIN')|
all_params['AnnualVariableOperatingCost'].t.str.startswith('RNW')].drop('r', axis=1)
dom_val_df = df_filter(dom_val_df,3,6,[])
for each in dom_val_df.columns:
if each in ['Land','Water','Geothermal','Hydro','Solar','Wind', 'Precipitation']:
dom_val_df = dom_val_df.drop(each, axis=1)
imp_val_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('IMP')].drop('r', axis=1)
imp_val_df = df_filter(imp_val_df,3,6,[])
exp_val_df = all_params['AnnualVariableOperatingCost'][all_params['AnnualVariableOperatingCost'].t.str.startswith('EXP')].drop('r', axis=1)
exp_val_df = df_filter(exp_val_df,3,6,[])
temp_col_list = []
temp_col_list = dom_val_df.columns
if len(imp_val_df.columns) > 1:
temp_col_list = temp_col_list.append(imp_val_df.columns)
if len(exp_val_df.columns) > 1:
temp_col_list = temp_col_list.append(exp_val_df.columns)
fue_val_df = pd.DataFrame(columns=list(set(temp_col_list)))
fue_val_df['y'] = years
fue_val_df = fue_val_df.reindex(sorted(fue_val_df.columns),axis=1).set_index('y').reset_index()
fue_val_df = fue_val_df.fillna(0)
for each in dom_val_df.columns:
if each != 'y':
fue_val_df[each] = dom_val_df[each]
fue_val_df = fue_val_df.fillna(0)
for each in imp_val_df.columns :
if each != 'y' and len(imp_val_df.columns) > 1:
fue_val_df[each] = fue_val_df[each] + imp_val_df[each]
fue_val_df = fue_val_df.fillna(0)
for each in exp_val_df.columns:
if each != 'y' and len(ene_exp_df.columns) > 1:
fue_val_df[each] = fue_val_df[each] + exp_val_df[each]
fue_val_df = fue_val_df.fillna(0)
temp_col_list = []
temp_col_list = dom_prd_df.columns
if len(ene_imp_df.columns) > 1:
temp_col_list = temp_col_list.append(ene_imp_df.columns)
if len(ene_exp_df.columns) > 1:
temp_col_list = temp_col_list.append(ene_exp_df.columns)
fue_prd_df = pd.DataFrame(columns=list(set(temp_col_list)))
fue_prd_df['y'] = years
fue_prd_df = fue_prd_df.reindex(sorted(fue_prd_df.columns),axis=1).set_index('y').reset_index()
fue_prd_df = fue_prd_df.fillna(0)
for each in dom_prd_df.columns:
if each != 'y':
fue_prd_df[each] = dom_prd_df[each]
fue_prd_df = fue_prd_df.fillna(0)
for each in ene_imp_df.columns:
if each != 'y' and len(ene_imp_df.columns) > 1:
fue_prd_df[each] = fue_prd_df[each] + ene_imp_df[each]
fue_prd_df = fue_prd_df.fillna(0)
for each in ene_exp_df.columns:
if each != 'y' and len(ene_exp_df.columns) > 1:
fue_prd_df[each] = fue_prd_df[each] - ene_exp_df[each]
fue_prd_df = fue_prd_df.fillna(0)
for df in [fue_val_df, fue_prd_df]:
#df['Diesel'] = df['Crude oil'].mul(0.3755) + df['Diesel']
#df['HFO'] = df['Crude oil'].mul(0.0171) + df['HFO']
#df.drop('Crude oil', axis=1, inplace=True)
df['Diesel'] = df['Diesel']
df['HFO'] = df['HFO']
#df.drop('Crude oil', axis=1, inplace=True)
fue_cos_df = pd.DataFrame(columns=list(set(temp_col_list)))
fue_cos_df['y'] = years
fue_cos_df = (fue_val_df/fue_prd_df)*gen_use_df
fue_cos_df = fue_cos_df.fillna(0)
fue_cos_df = fue_cos_df.reindex(sorted(fue_cos_df.columns),axis=1).set_index('y').reset_index()
fue_cos_df['y'] = years
ele_cos_df['Electricity generation'] = gen_df.iloc[:,1:].sum(axis=1)/3.6
ele_cos_df['Capital costs'] = ele_cos_df['Capital costs']/ele_cos_df['Electricity generation']
ele_cos_df['Fixed costs'] = fix_cos_df.iloc[:,1:].sum(axis=1)/ele_cos_df['Electricity generation']
ele_cos_df['Variable costs'] = var_cos_df.iloc[:,1:].sum(axis=1)/ele_cos_df['Electricity generation']
ele_cos_df['Fuel distribution costs'] = dis_cos_df.iloc[:,1:].sum(axis=1)/ele_cos_df['Electricity generation']
ele_cos_df['Fuel costs'] = fue_cos_df.iloc[:,1:].sum(axis=1)/ele_cos_df['Electricity generation']
ele_cos_df.drop('Electricity generation',axis=1,inplace=True)
ele_cos_df.iplot(kind='bar',barmode='relative',x='y',title='Cost of electricity generation ($/MWh)')
# Land use
regions = {"TIG": 'TIG',
"AFA": 'AFA',
"AMH": 'AMH',
"BEN": 'BEN',
#"DIR": '0',
#"HAR",
#"ADD",
"GAM": 'GAM',
"ORO": 'ORO',
"SOM": 'SOM',
"SOU": 'SOU',
}
crops = {}
for each in det_col.keys():
if each.startswith('CP'):
crops[each] = det_col[each]
# DEFINE THAT THE NAMING CONVENTION FILE NEED TO LIST CROPS ALPHABETICAL
water_supply = {'I':'Irrigated',
'R':'Rain-fed'}
input_level = {'L':'Low',
'I':'Intermediate',
'H':'High'}
mode_crop_combo_input = pd.read_csv(os.path.join(os.getcwd(),'mode_list.csv'), encoding='latin-1')
mode_crop_combo = dict([(m,c) for m,c in zip(mode_crop_combo_input['mode'], mode_crop_combo_input['crop_combo'])])
crops_total_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)
#crops_total_df['land_use'] = crops_total_df.m.map(mode_search)
crops_total_df['m'] = crops_total_df['m'].astype(int)
crops_total_df['crop_combo'] = crops_total_df['m'].map(mode_crop_combo)
crops_total_df['land_use'] = crops_total_df['crop_combo'].str[0:4]
crops_total_df.drop(['m','crop_combo'], axis=1, inplace=True)
crops_total_df = crops_total_df[crops_total_df['land_use'].str.startswith('CP')]
crops_total_df = crops_total_df.pivot_table(index='y',
columns='land_use',
values='value',
aggfunc='sum').reset_index().fillna(0)
crops_total_df = crops_total_df.reindex(sorted(crops_total_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col).astype('float64')
if not crops_total_df.empty:
df_plot(crops_total_df,'Land area (1000 sq.km.)','Area by crop')
land_total_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)
land_total_df['m'] = land_total_df['m'].astype(int)
land_total_df['crop_combo'] = land_total_df['m'].map(mode_crop_combo)
land_total_df['land_use'] = land_total_df['crop_combo'].str[0:4]
land_total_df.drop(['m','crop_combo'], axis=1, inplace=True)
land_total_df = land_total_df.pivot_table(index='y',
columns='land_use',
values='value',
aggfunc='sum').reset_index().fillna(0)
land_total_df['AGR'] = 0
for crop in crops:
if crop in land_total_df.columns:
land_total_df['AGR'] += land_total_df[crop]
land_total_df.drop(crop, axis=1, inplace=True)
land_total_df = land_total_df.reindex(sorted(land_total_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col).astype('float64')
if not land_total_df.empty:
df_plot(land_total_df,'Land area (1000 sq.km.)','Area by land cover type')
for each in regions.keys():
crops_region_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)
crops_region_df = crops_region_df[crops_region_df.t.str[6:9] == each]
crops_region_df['m'] = crops_region_df['m'].astype(int)
crops_region_df['crop_combo'] = crops_region_df['m'].map(mode_crop_combo)
crops_region_df['land_use'] = crops_region_df['crop_combo'].str[0:4]
crops_region_df.drop(['m','crop_combo'], axis=1, inplace=True)
crops_region_df = crops_region_df[crops_region_df['land_use'].str.startswith('CP')]
crops_region_df = crops_region_df.pivot_table(index='y',
columns='land_use',
values='value',
aggfunc='sum').reset_index().fillna(0)
crops_region_df = crops_region_df.reindex(sorted(crops_region_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col).astype('float64')
if not crops_region_df.empty:
df_plot(crops_region_df,'Land area (1000 sq.km.)','Area by crop (' + regions[each] + ' region)')
for each in regions.keys():
land_cluster_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)
land_cluster_df = land_cluster_df[land_cluster_df.t.str[6:9] == each]
land_cluster_df['m'] = land_cluster_df['m'].astype(int)
land_cluster_df['crop_combo'] = land_cluster_df['m'].map(mode_crop_combo)
land_cluster_df['land_use'] = land_cluster_df['crop_combo'].str[0:4]
land_cluster_df.drop(['m','crop_combo'], axis=1, inplace=True)
land_cluster_df['value'] = land_cluster_df['value'].astype('float64')
land_cluster_df = land_cluster_df.pivot_table(index='y',
columns='land_use',
values='value',
aggfunc='sum').reset_index().fillna(0)
land_cluster_df['AGR'] = 0
for crop in crops:
if crop in land_cluster_df.columns:
land_cluster_df['AGR'] += land_cluster_df[crop]
land_cluster_df.drop(crop, axis=1, inplace=True)
land_cluster_df = land_cluster_df.reindex(sorted(land_cluster_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
if not land_cluster_df.empty:
df_plot(land_cluster_df,'Land area (1000 sq.km.)','Area by land cover type (' + regions[each] + ' region)')
for each in water_supply.keys():
crops_ws_df = all_params['TotalAnnualTechnologyActivityByMode'][all_params['TotalAnnualTechnologyActivityByMode'].t.str.startswith('LNDAGR')].drop('r', axis=1)
crops_ws_df['m'] = crops_ws_df['m'].astype(int)
crops_ws_df['crop_combo'] = crops_ws_df['m'].map(mode_crop_combo)
#print(crops_ws_df.head())
crops_ws_df = crops_ws_df[(crops_ws_df.crop_combo.str.startswith('CP')) & (crops_ws_df.crop_combo.str[5:6] == each)]
crops_ws_df['land_use'] = crops_ws_df['crop_combo'].str[0:4]
crops_ws_df.drop(['m','crop_combo'], axis=1, inplace=True)
crops_ws_df = crops_ws_df.pivot_table(index='y',
columns='land_use',
values='value',
aggfunc='sum').reset_index().fillna(0)
crops_ws_df = crops_ws_df.reindex(sorted(crops_ws_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
if len(crops_ws_df.columns) > 1:
df_plot(crops_ws_df,'Land area (1000 sq.km.)','Area by crop (' + water_supply[each] + ')')
crops_prod_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str.startswith('CRP')].drop('r', axis=1)
crops_prod_df['f'] = crops_prod_df['f'].str[3:7]
crops_prod_df['value'] = crops_prod_df['value'].astype('float64')
crops_prod_df = crops_prod_df.pivot_table(index='y',
columns='f',
values='value',
aggfunc='sum').reset_index().fillna(0)
crops_prod_df = crops_prod_df.reindex(sorted(crops_prod_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
crops_prod_df['y'] = years
if len(crops_prod_df.columns) > 1:
df_plot(crops_prod_df,'Production (Million tonnes)','Crop production')
lvs_prod_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str.startswith('LVS')].drop('r', axis=1)
lvs_prod_df = lvs_prod_df.loc[lvs_prod_df['f'].str.contains('PRD')]
lvs_prod_df['value'] = lvs_prod_df['value'].astype('float64')
lvs_prod_df = lvs_prod_df.pivot_table(index='y',
columns='f',
values='value',
aggfunc='sum').reset_index().fillna(0)
lvs_prod_df = lvs_prod_df.reindex(sorted(lvs_prod_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
lvs_prod_df['y'] = years
if len(lvs_prod_df.columns) > 1:
df_plot(lvs_prod_df,'Population (Million head)','Livestock production')
crops_yield_df = crops_prod_df/crops_total_df
crops_yield_df['y'] = years
#print(crops_yield_df.loc[crops_yield_df['y'] == min(years)].transpose().to_string(header=False, index=False))
crops_yield_df.loc[:,crops_yield_df.columns != 'y'] = crops_yield_df.loc[:,crops_yield_df.columns != 'y'].mul(10)
if len(crops_yield_df.columns) > 1:
crops_yield_df.iplot(x='y',
mode='lines+markers',
xTitle='Year',
yTitle='Yield (t/ha)',
size=10,
color=[color_dict[x] for x in crops_yield_df.columns if x != 'y'],
title='Yield (tonnes/hectare)')#
wat_dem_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str[0:6].isin(['AGRWAT','PUBWAT','PWRWAT','INDWAT','LVSWAT'])].drop('r', axis=1)
wat_dem_df['f'] = wat_dem_df['f'].str[0:3]
wat_dem_df['value'] = wat_dem_df['value'].astype('float64')
wat_dem_df = wat_dem_df.pivot_table(index='y',
columns='f',
values='value',
aggfunc='sum').reset_index().fillna(0)
pwrwat_df = all_params['ProductionByTechnologyAnnual'][(all_params['ProductionByTechnologyAnnual'].t.str.startswith('PWR')) &
all_params['ProductionByTechnologyAnnual'].f.str.startswith('WTR')].drop('r', axis=1)
pwrwat_df['f'] = pwrwat_df['f'].str[0:3]
pwrwat_df['value'] = pwrwat_df['value'].astype('float64')
pwrwat_df = pwrwat_df.pivot_table(index='y',
columns='f',
values='value',
aggfunc='sum').reset_index().fillna(0)
if not pwrwat_df.empty:
wat_dem_df['PWR_WAT'] = wat_dem_df['PWR'] - pwrwat_df['WTR']
wat_dem_df = wat_dem_df.reindex(sorted(wat_dem_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
if 'Power sector' in wat_dem_df.columns:
wat_dem_df.rename(columns={'Power sector':'Power sector (withdrawal)'}, inplace=True)
wat_dem_df = wat_dem_df.loc[:, (wat_dem_df != 0).any(axis=0)]
wat_dem_df.iplot(x='y',
kind='bar',
barmode='stack',
xTitle='Year',
yTitle='Billion m3',
color=[color_dict[x] for x in wat_dem_df.columns if x != 'y'],
title='Water Demand',
showlegend=True)
wat_source_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str[0:6].isin(['AGRWAT','PUBWAT','PWRWAT'])].drop('r', axis=1)
wat_source_df = df_filter(wat_source_df,6,9,[])
df_plot(wat_source_df, 'Billion m3', 'Water withdrawal by source')
wat_bal_df = all_params['ProductionByTechnologyAnnual'][all_params['ProductionByTechnologyAnnual'].f.str.startswith('WTR')].drop('r', axis=1)
wat_bal_df['f'] = wat_bal_df['f'].str[3:6]
wat_bal_df['value'] = wat_bal_df['value'].astype('float64')
wat_bal_df = wat_bal_df.pivot_table(index='y',
columns='f',
values='value',
aggfunc='sum').reset_index().fillna(0)
wat_bal_df = wat_bal_df.reindex(sorted(wat_bal_df.columns), axis=1).set_index('y').reset_index().rename(columns=det_col)
wat_bal_df['Irrigation'] = wat_dem_df['Agriculture']
wat_bal_df['y'] = years
for each in wat_bal_df.columns:
if each in ['Evapotranspiration', 'Groundwater recharge', 'Surface water run-off']:
wat_bal_df[each] = wat_bal_df[each].mul(-1)
if 'Power sector (withdrawal)' in wat_bal_df:
wat_bal_df['Surface water run-off'] = wat_bal_df['Surface water run-off'] + wat_dem_df['Power sector (withdrawal)'] - wat_dem_df['Power sector (consumptive use)']
wat_bal_df.iplot(x='y',
kind='bar',
barmode='relative',
xTitle='Year',
yTitle='Billion m3',
color=[color_dict[x] for x in wat_bal_df.columns if x != 'y'],
title='Water balance')